package com.study.bigdata.spark.core.rdd.operator.transform

import org.apache.spark.{SparkConf, SparkContext}

object Scala06_RDD_Operator_Transform_2 {
  def main(args: Array[String]): Unit = {

    val conf = new SparkConf().setMaster("local[*]").setAppName("RDD")
    conf.set("spark.local.dir","D:\\hadoopbook\\spark\\test")
    val sc = new SparkContext(conf)

    // 从服务器日志数据apache.log中获取每个时间段访问量
    // TODO groupBy算子可以实现WordCount (1)
    val lines = sc.textFile("data/apache.log")
    val rdd = lines.map(
      lines => {
        val datas = lines.split(" ")
        val time = datas(3)
        val times = time.split(":")
        (times(1), 1)
      }
    ).groupBy(_._1)
    val rdd1 = rdd.mapValues(_.size)
    rdd1.collect().foreach(println)
    /*
    (05,371)
    (01,360)
    (10,443)
    (23,356)......
     */
    sc.stop()

  }

}
